package com.xiaotu.spider.project.pageprocessor.news;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.*;
import com.xiaotu.spider.AUrlPageProcessor;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.math.BigDecimal;
import java.util.*;

/**
 * 爬取今日头条的新闻
 */
@Component("NewsTouTiaoPageProcessor")
public class NewsTouTiaoPageProcessor extends AUrlPageProcessor
{
    private static final Logger LOGGER = LoggerFactory.getLogger(NewsTouTiaoPageProcessor.class);

    private static final String URL = "https://www.toutiao.com" +
            "/api/pc/feed/?category=news_%s&utm_source=toutiao&widen=1&max_behot_time=0&max_behot_time_tmp=0" +
            "&tadrequire=true&as=A145A9C9F6AC101&cp=59960CA120913E1";

    private static final String DETAIL_URL = "https://www.toutiao.com/a%s/";

    private static final String[] CATEGORYS = new String[]{"entertainment", "tech", "sports", "car", "finance",
            "travel", "baby", "food", "regimen", "history", "world", "game"};

    private static final String[] TYPES = new String[]{"娱乐", "科技", "体育", "汽车", "财经", "旅游", "育儿", "美食", "养生", "历史",
            "国际", "游戏"};

    private String filePath;

    private String newsType;

    private int maxDays = 3;

    @Override
    public void process(Page page)
    {
        String url = page.getUrl().toString();
        if (url.contains("max_behot_time") && url.contains("max_behot_time_tmp"))
            this.listProcess(page);
        else
            this.detailProcess(page);
    }

    private void detailProcess(Page page)
    {
        String response = page.getRawText();
        if (StringUtils.isEmpty(response))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        response = RegexUtils.regexSearch("content: '([^']+)'", response);
        if (StringUtils.isEmpty(response))
            return;
        response = response.substring("content: '".length(), response.lastIndexOf("'"));
        response = StringEscapeUtils.unescapeHtml(response);
        response = HtmlUtil.delHTMLTag(response);
        if (StringUtils.isEmpty(response))
            return;
        String fileName = this.getFileName(page, RegexUtils.regexSearch("a" + RegexUtils.REGEX_INTEGER, page.getUrl()
                .toString()));
        FileUtil.writeFile(fileName, response);
        LOGGER.debug("{} writed success!", fileName);
    }

    private void listProcess(Page page)
    {
        String json = page.getRawText();
        if (StringUtils.isEmpty(json))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Map<String, Object> map = GsonUtils.fromJson(json, Map.class);
        List<Map<String, Object>> dataList = (List<Map<String, Object>>) map.get("data");
        if (dataList == null || dataList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        dataList.stream().filter(dataMap ->
        {
            String fileName = this.getFileName(page, "a" + dataMap.get("group_id"));
            boolean isExits = FileUtil.fileExits(fileName);
            if (isExits)
                LOGGER.debug("{} exits", fileName);
            return !isExits;
        }).forEach(dataMap -> PageUtils.addTargetRequest(page, String.format(DETAIL_URL, dataMap.get("group_id")),
                page.getRequest().getExtras()));

        Map<String, Object> nextMap = (Map<String, Object>) map.get("next");
        if (nextMap != null && nextMap.containsKey("max_behot_time"))
        {
            String url = page.getUrl().toString();
            String time = StringUtil.nullToString(nextMap.get("max_behot_time"));
            if (StringUtils.isEmpty(time))
                return;
            BigDecimal val = new BigDecimal(time);
            if (!this.isRightTime(val))
                return;

            time = String.valueOf(val.longValue());
            url = url.replaceAll("max_behot_time=(" + RegexUtils.REGEX_INTEGER + ")&max_behot_time_tmp=(" +
                    RegexUtils.REGEX_INTEGER + ")", "max_behot_time=" + time + "&max_behot_time_tmp=" + time);
            PageUtils.addTargetRequest(page, url, page.getRequest().getExtras());
        }
    }

    private boolean isRightTime(BigDecimal val)
    {
        Date newsDate = new Date(val.longValue() * 1000);
        return new Date().before(DateUtils.addInteger(newsDate, Calendar.DAY_OF_YEAR, this.maxDays));
    }

    private String getFileName(Page page, String groupId)
    {
        return this.filePath + page.getRequest().getExtra("type") + File.separator + groupId + ".txt";
    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        String path = StringUtil.nullToString(map.get("path"));
        if (StringUtils.isEmpty(path))
            throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
        this.filePath = path;
        this.newsType = StringUtil.nullToString(map.get("newsType"));
        String days = StringUtil.nullToString(map.get("maxDays"));
        if (StringUtils.isNotEmpty(days))
            this.maxDays = Integer.parseInt(days);
    }

    @Override
    protected List<JSONObject> getStartRequestUrlsAndParams()
    {
        List<JSONObject> list = new ArrayList<>();
        for (int i = 0; i < TYPES.length; i++)
        {
            if (StringUtils.isNotEmpty(newsType) && !newsType.contains(TYPES[i]))
                continue;
            JSONObject json = new JSONObject();
            json.put(URL_KEY, String.format(URL, CATEGORYS[i]));
            json.put("type", TYPES[i]);
            list.add(json);
        }
        return list;
    }

    @Override
    protected String[] getStartRequestUrls()
    {
        return null;
    }

    @Override
    protected String getSpiderType()
    {
        return "NewsTouTiao";
    }

    @Override
    public String getPageDataType(Page page)
    {
        return Constants.PageDataType.SEARCH_LIST;
    }
}
